package com.zhanggm.s3.house.webmagic;

import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.HashMap;
import java.util.List;

public class BendibaoSpider implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000)
            .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36");

    @Override
    public void process(Page page) {
        //视频标题
        List<Selectable> trList = page.getHtml().xpath("//table/tbody/tr").nodes();
        trList.forEach(item->{
            List<Selectable> tdList = item.xpath("//td").nodes();
            System.out.println(tdList.size());
            HashMap<String, String> stringStringHashMap = new HashMap<>();
            //<td bgcolor="#ebebeb" width="10%"><strong>区</strong></td>
            stringStringHashMap.put("f1", tdList.get(tdList.size()-1).xpath("//allText()").get());
            stringStringHashMap.put("f2", tdList.get(tdList.size()-2).xpath("//allText()").get());
            stringStringHashMap.put("f3", tdList.get(tdList.size()-3).xpath("//allText()").get());
            stringStringHashMap.put("f4", tdList.get(tdList.size()-4).xpath("//allText()").get());

            System.out.println(stringStringHashMap);
        });
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
            Spider.create(new BendibaoSpider()).addUrl("http://bj.bendibao.com/cyfw/201337/97497.shtm")
                    //传输到数据库
                     .addPipeline(new Pipeline() {
                         @Override
                         public void process(ResultItems resultItems, Task task) {
                            
                         }
                     })
                    //开启5个线程抓取
                    .thread(1)
                    //启动爬虫
                    .run();


    }
}
